{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import gzip\n",
    "import numpy\n",
    "from __future__ import print_function\n",
    "import tensorflow as tf\n",
    "import theano.tensor as T\n",
    "import theano.tensor\n",
    "#import _pickle as cPickle\n",
    "import os\n",
    "os.chdir('D:\\\\Aegis\\\\Deep Learning')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting gz file into pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\" with gzip.open('17flower.gz', 'rb') as f:\\n        flower = list(pickle.load(f))\\n        write_file=open('flower.pkl','wb')\\n        write_file.close() \""
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' with gzip.open('17flower.gz', 'rb') as f:\n",
    "        flower = list(pickle.load(f))\n",
    "        write_file=open('flower.pkl','wb')\n",
    "        write_file.close() '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_file = 'flower.pkl'\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    train_set, valid_set, test_set = pickle.load(f, encoding='latin1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shared_dataset(data_xy):\n",
    "\n",
    "    data_x, data_y = data_xy\n",
    "\n",
    "    shared_x = theano.shared(numpy.asarray(data_x, dtype=theano.config.floatX))\n",
    "\n",
    "    shared_y = theano.shared(numpy.asarray(data_y, dtype=theano.config.floatX))\n",
    "\n",
    "    return shared_x, T.cast(shared_y, 'int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(680, 784)\n",
      "(680,)\n",
      "(339, 784)\n"
     ]
    }
   ],
   "source": [
    "test_set_x, test_set_y0 = shared_dataset(test_set)\n",
    "\n",
    "valid_set_x, valid_set_y0 = shared_dataset(valid_set)\n",
    "\n",
    "train_set_x, train_set_y0 = shared_dataset(train_set)\n",
    "\n",
    "print(train_set_x.eval().shape)\n",
    "\n",
    "print(train_set_y0.eval().shape)\n",
    "\n",
    "print(valid_set_x.eval().shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "\n",
    "training_iters = 250000\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "display_step = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "\n",
    "n_classes = 17 # MNIST total classes (0-9 digits)\n",
    "\n",
    "dropout = 0.75 # Dropout, probability to keep units"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## change_train_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "680\n"
     ]
    }
   ],
   "source": [
    "train_set_y1 = [[0 for col in range(n_classes)] for row in range(train_set_x.eval().shape[0])]\n",
    "print(len(train_set_y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(train_set_y0.eval().shape[0]):\n",
    "\n",
    "    temp = train_set_y0.eval()[i]\n",
    "\n",
    "    train_set_y1[i][temp] = 1\n",
    "\n",
    "shared_y1 = theano.shared(numpy.asarray(train_set_y1, dtype=theano.config.floatX))\n",
    "\n",
    "train_set_y = T.cast(shared_y1, 'float64')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## change_valid_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "339\n"
     ]
    }
   ],
   "source": [
    "valid_set_y1 = [[0 for col in range(n_classes)] for row in range(valid_set_x.eval().shape[0])]\n",
    "print(len(valid_set_y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(valid_set_y0.eval().shape[0]):\n",
    "\n",
    "    temp = valid_set_y0.eval()[i]\n",
    "\n",
    "    valid_set_y1[i][temp] = 1\n",
    "\n",
    "shared_y1 = theano.shared(numpy.asarray(valid_set_y1, dtype=theano.config.floatX))\n",
    "\n",
    "valid_set_y = T.cast(shared_y1, 'float64')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## change_test_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "339\n"
     ]
    }
   ],
   "source": [
    "test_set_y1 = [[0 for col in range(n_classes)] for row in range(test_set_x.eval().shape[0])]\n",
    "print(len(test_set_y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(test_set_y0.eval().shape[0]):\n",
    "\n",
    "    temp = test_set_y0.eval()[i]\n",
    "\n",
    "    test_set_y1[i][temp] = 1\n",
    "\n",
    "shared_y1 = theano.shared(numpy.asarray(test_set_y1, dtype=theano.config.floatX))\n",
    "\n",
    "test_set_y = T.cast(shared_y1, 'float64')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tf Graph input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Placeholder_8:0\", dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "\n",
    "y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "\n",
    "keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)\n",
    "print(keep_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create some wrappers for simplicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv2d(x, W, b, strides=1):\n",
    "\n",
    "    # Conv2D wrapper, with bias and relu activation\n",
    "\n",
    "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
    "\n",
    "    x = tf.nn.bias_add(x, b)\n",
    "\n",
    "    return tf.nn.relu(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maxpool2d(x, k=2):\n",
    "\n",
    "    # MaxPool2D wrapper\n",
    "\n",
    "    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],\n",
    "\n",
    "                          padding='SAME')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_net(x, weights, biases, dropout):\n",
    "\n",
    "    # Reshape input picture\n",
    "\n",
    "    x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
    "    \n",
    "\n",
    "    # Convolution Layer\n",
    "\n",
    "    conv1 = conv2d(x, weights['wc1'], biases['bc1'])\n",
    "\n",
    "    # Max Pooling (down-sampling)\n",
    "\n",
    "    conv1 = maxpool2d(conv1, k=2)\n",
    "\n",
    "\n",
    "\n",
    "    # Convolution Layer\n",
    "\n",
    "    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])\n",
    "\n",
    "    # Max Pooling (down-sampling)\n",
    "\n",
    "    conv2 = maxpool2d(conv2, k=2)\n",
    "\n",
    "\n",
    "\n",
    "    # Fully connected layer\n",
    "\n",
    "    # Reshape conv2 output to fit fully connected layer input\n",
    "\n",
    "    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])\n",
    "\n",
    "    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])\n",
    "\n",
    "    fc1 = tf.nn.relu(fc1)\n",
    "\n",
    "    # Apply Dropout\n",
    "\n",
    "    fc1 = tf.nn.dropout(fc1, dropout)\n",
    "\n",
    "\n",
    "\n",
    "    # Output, class prediction\n",
    "\n",
    "    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])\n",
    "\n",
    "    return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Store layers weight & bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = {\n",
    "\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32])),\n",
    "\n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64])),\n",
    "\n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024])),\n",
    "\n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "\n",
    "    'out': tf.Variable(tf.random_normal([1024, n_classes]))\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "biases = {\n",
    "\n",
    "    'bc1': tf.Variable(tf.random_normal([32])),\n",
    "\n",
    "    'bc2': tf.Variable(tf.random_normal([64])),\n",
    "\n",
    "    'bd1': tf.Variable(tf.random_normal([1024])),\n",
    "\n",
    "    'out': tf.Variable(tf.random_normal([n_classes]))\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = conv_net(x, weights, biases, keep_prob)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define loss and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Mean_5:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initializing the variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "\n",
    "valid_num = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_try\n",
      "Iter 1280, Minibatch Loss= 23761.435547, Training Accuracy= 0.07031\n",
      "Iter 1280, Minibatch Loss= 21258.011719, Valid Accuracy= 0.09375\n",
      "Iter 2560, Minibatch Loss= 14560.416992, Training Accuracy= 0.17188\n",
      "Iter 2560, Minibatch Loss= 15658.187500, Valid Accuracy= 0.10938\n",
      "Iter 3840, Minibatch Loss= 11073.394531, Training Accuracy= 0.15625\n",
      "Iter 3840, Minibatch Loss= 13585.617188, Valid Accuracy= 0.12500\n",
      "Iter 5120, Minibatch Loss= 10343.902344, Training Accuracy= 0.25000\n",
      "Iter 5120, Minibatch Loss= 12755.626953, Valid Accuracy= 0.18750\n",
      "Iter 6400, Minibatch Loss= 8167.491699, Training Accuracy= 0.22656\n",
      "Iter 6400, Minibatch Loss= 10336.007812, Valid Accuracy= 0.17188\n",
      "Iter 7680, Minibatch Loss= 6807.632812, Training Accuracy= 0.27344\n",
      "Iter 7680, Minibatch Loss= 8238.911133, Valid Accuracy= 0.20312\n",
      "Iter 8960, Minibatch Loss= 6044.257812, Training Accuracy= 0.29688\n",
      "Iter 8960, Minibatch Loss= 7378.135742, Valid Accuracy= 0.20312\n",
      "Iter 10240, Minibatch Loss= 5412.616699, Training Accuracy= 0.32812\n",
      "Iter 10240, Minibatch Loss= 7035.746094, Valid Accuracy= 0.25000\n",
      "Iter 11520, Minibatch Loss= 4439.116699, Training Accuracy= 0.32812\n",
      "Iter 11520, Minibatch Loss= 6011.810547, Valid Accuracy= 0.29688\n",
      "Iter 12800, Minibatch Loss= 3738.923828, Training Accuracy= 0.39844\n",
      "Iter 12800, Minibatch Loss= 4944.385254, Valid Accuracy= 0.29688\n",
      "Iter 14080, Minibatch Loss= 3450.482910, Training Accuracy= 0.39844\n",
      "Iter 14080, Minibatch Loss= 3937.887207, Valid Accuracy= 0.35938\n",
      "Iter 15360, Minibatch Loss= 3032.173584, Training Accuracy= 0.43750\n",
      "Iter 15360, Minibatch Loss= 4119.644043, Valid Accuracy= 0.35938\n",
      "Iter 16640, Minibatch Loss= 2473.374268, Training Accuracy= 0.47656\n",
      "Iter 16640, Minibatch Loss= 3755.496094, Valid Accuracy= 0.39062\n",
      "Iter 17920, Minibatch Loss= 2267.738281, Training Accuracy= 0.50781\n",
      "Iter 17920, Minibatch Loss= 3396.093262, Valid Accuracy= 0.40625\n",
      "Iter 19200, Minibatch Loss= 1915.803223, Training Accuracy= 0.54688\n",
      "Iter 19200, Minibatch Loss= 2716.124512, Valid Accuracy= 0.43750\n",
      "Iter 20480, Minibatch Loss= 1723.488647, Training Accuracy= 0.57812\n",
      "Iter 20480, Minibatch Loss= 2536.899170, Valid Accuracy= 0.46875\n",
      "Iter 21760, Minibatch Loss= 1608.083008, Training Accuracy= 0.57812\n",
      "Iter 21760, Minibatch Loss= 2324.376953, Valid Accuracy= 0.48438\n",
      "Iter 23040, Minibatch Loss= 1259.798584, Training Accuracy= 0.63281\n",
      "Iter 23040, Minibatch Loss= 2444.336914, Valid Accuracy= 0.46875\n",
      "Iter 24320, Minibatch Loss= 1364.543213, Training Accuracy= 0.57812\n",
      "Iter 24320, Minibatch Loss= 2437.636719, Valid Accuracy= 0.42188\n",
      "Iter 25600, Minibatch Loss= 1066.329468, Training Accuracy= 0.64844\n",
      "Iter 25600, Minibatch Loss= 1997.766846, Valid Accuracy= 0.46875\n",
      "Iter 26880, Minibatch Loss= 1031.933960, Training Accuracy= 0.62500\n",
      "Iter 26880, Minibatch Loss= 1950.067871, Valid Accuracy= 0.46875\n",
      "Iter 28160, Minibatch Loss= 833.383850, Training Accuracy= 0.68750\n",
      "Iter 28160, Minibatch Loss= 1831.902588, Valid Accuracy= 0.51562\n",
      "Iter 29440, Minibatch Loss= 723.658447, Training Accuracy= 0.71875\n",
      "Iter 29440, Minibatch Loss= 1718.573486, Valid Accuracy= 0.48438\n",
      "Iter 30720, Minibatch Loss= 707.461914, Training Accuracy= 0.74219\n",
      "Iter 30720, Minibatch Loss= 1417.382690, Valid Accuracy= 0.57812\n",
      "Iter 32000, Minibatch Loss= 544.239624, Training Accuracy= 0.75781\n",
      "Iter 32000, Minibatch Loss= 1522.660645, Valid Accuracy= 0.53125\n",
      "Iter 33280, Minibatch Loss= 479.209686, Training Accuracy= 0.77344\n",
      "Iter 33280, Minibatch Loss= 1458.229736, Valid Accuracy= 0.56250\n",
      "Iter 34560, Minibatch Loss= 413.075470, Training Accuracy= 0.77344\n",
      "Iter 34560, Minibatch Loss= 1655.687744, Valid Accuracy= 0.53125\n",
      "Iter 35840, Minibatch Loss= 420.159271, Training Accuracy= 0.81250\n",
      "Iter 35840, Minibatch Loss= 1385.559814, Valid Accuracy= 0.57812\n",
      "Iter 37120, Minibatch Loss= 339.929932, Training Accuracy= 0.83594\n",
      "Iter 37120, Minibatch Loss= 1065.049316, Valid Accuracy= 0.64062\n",
      "Iter 38400, Minibatch Loss= 261.439484, Training Accuracy= 0.85156\n",
      "Iter 38400, Minibatch Loss= 1328.357666, Valid Accuracy= 0.59375\n",
      "Iter 39680, Minibatch Loss= 237.947144, Training Accuracy= 0.87500\n",
      "Iter 39680, Minibatch Loss= 1331.871338, Valid Accuracy= 0.59375\n",
      "Iter 40960, Minibatch Loss= 216.418213, Training Accuracy= 0.85938\n",
      "Iter 40960, Minibatch Loss= 1305.859863, Valid Accuracy= 0.65625\n",
      "Iter 42240, Minibatch Loss= 194.594238, Training Accuracy= 0.83594\n",
      "Iter 42240, Minibatch Loss= 1078.983154, Valid Accuracy= 0.67188\n",
      "Iter 43520, Minibatch Loss= 172.766235, Training Accuracy= 0.85156\n",
      "Iter 43520, Minibatch Loss= 1017.426331, Valid Accuracy= 0.68750\n",
      "Iter 44800, Minibatch Loss= 160.811707, Training Accuracy= 0.91406\n",
      "Iter 44800, Minibatch Loss= 1117.873779, Valid Accuracy= 0.65625\n",
      "Iter 46080, Minibatch Loss= 148.815765, Training Accuracy= 0.89844\n",
      "Iter 46080, Minibatch Loss= 1011.640686, Valid Accuracy= 0.65625\n",
      "Iter 47360, Minibatch Loss= 150.736176, Training Accuracy= 0.91406\n",
      "Iter 47360, Minibatch Loss= 978.887512, Valid Accuracy= 0.70312\n",
      "Iter 48640, Minibatch Loss= 114.550568, Training Accuracy= 0.92188\n",
      "Iter 48640, Minibatch Loss= 888.321106, Valid Accuracy= 0.73438\n",
      "Iter 49920, Minibatch Loss= 73.420036, Training Accuracy= 0.94531\n",
      "Iter 49920, Minibatch Loss= 946.508423, Valid Accuracy= 0.68750\n",
      "Iter 51200, Minibatch Loss= 75.674301, Training Accuracy= 0.94531\n",
      "Iter 51200, Minibatch Loss= 910.363281, Valid Accuracy= 0.71875\n",
      "Iter 52480, Minibatch Loss= 77.244362, Training Accuracy= 0.94531\n",
      "Iter 52480, Minibatch Loss= 768.624573, Valid Accuracy= 0.73438\n",
      "Iter 53760, Minibatch Loss= 112.038147, Training Accuracy= 0.95312\n",
      "Iter 53760, Minibatch Loss= 658.237183, Valid Accuracy= 0.78125\n",
      "Iter 55040, Minibatch Loss= 73.279617, Training Accuracy= 0.94531\n",
      "Iter 55040, Minibatch Loss= 775.564331, Valid Accuracy= 0.71875\n",
      "Iter 56320, Minibatch Loss= 56.043209, Training Accuracy= 0.97656\n",
      "Iter 56320, Minibatch Loss= 735.228333, Valid Accuracy= 0.70312\n",
      "Iter 57600, Minibatch Loss= 71.661926, Training Accuracy= 0.96094\n",
      "Iter 57600, Minibatch Loss= 746.226318, Valid Accuracy= 0.70312\n",
      "Iter 58880, Minibatch Loss= 34.631382, Training Accuracy= 0.97656\n",
      "Iter 58880, Minibatch Loss= 613.933838, Valid Accuracy= 0.75000\n",
      "Iter 60160, Minibatch Loss= 68.723450, Training Accuracy= 0.97656\n",
      "Iter 60160, Minibatch Loss= 549.489990, Valid Accuracy= 0.79688\n",
      "Iter 61440, Minibatch Loss= 56.078697, Training Accuracy= 0.96094\n",
      "Iter 61440, Minibatch Loss= 710.663208, Valid Accuracy= 0.71875\n",
      "Iter 62720, Minibatch Loss= 52.987549, Training Accuracy= 0.95312\n",
      "Iter 62720, Minibatch Loss= 774.668945, Valid Accuracy= 0.70312\n",
      "Iter 64000, Minibatch Loss= 45.591251, Training Accuracy= 0.96875\n",
      "Iter 64000, Minibatch Loss= 637.990051, Valid Accuracy= 0.73438\n",
      "Iter 65280, Minibatch Loss= 57.299976, Training Accuracy= 0.96094\n",
      "Iter 65280, Minibatch Loss= 518.819824, Valid Accuracy= 0.78125\n",
      "Iter 66560, Minibatch Loss= 56.955246, Training Accuracy= 0.96094\n",
      "Iter 66560, Minibatch Loss= 511.498260, Valid Accuracy= 0.73438\n",
      "Iter 67840, Minibatch Loss= 56.640671, Training Accuracy= 0.96094\n",
      "Iter 67840, Minibatch Loss= 542.464844, Valid Accuracy= 0.75000\n",
      "Iter 69120, Minibatch Loss= 57.301975, Training Accuracy= 0.96094\n",
      "Iter 69120, Minibatch Loss= 520.930664, Valid Accuracy= 0.75000\n",
      "Iter 70400, Minibatch Loss= 43.120190, Training Accuracy= 0.96875\n",
      "Iter 70400, Minibatch Loss= 473.224915, Valid Accuracy= 0.75000\n",
      "Iter 71680, Minibatch Loss= 68.663155, Training Accuracy= 0.97656\n",
      "Iter 71680, Minibatch Loss= 302.718384, Valid Accuracy= 0.84375\n",
      "Iter 72960, Minibatch Loss= 38.323868, Training Accuracy= 0.97656\n",
      "Iter 72960, Minibatch Loss= 452.710419, Valid Accuracy= 0.71875\n",
      "Iter 74240, Minibatch Loss= 55.522751, Training Accuracy= 0.97656\n",
      "Iter 74240, Minibatch Loss= 475.450043, Valid Accuracy= 0.71875\n",
      "Iter 75520, Minibatch Loss= 42.620930, Training Accuracy= 0.96875\n",
      "Iter 75520, Minibatch Loss= 424.533997, Valid Accuracy= 0.73438\n",
      "Iter 76800, Minibatch Loss= 38.116005, Training Accuracy= 0.97656\n",
      "Iter 76800, Minibatch Loss= 342.346680, Valid Accuracy= 0.79688\n",
      "Iter 78080, Minibatch Loss= 19.169817, Training Accuracy= 0.98438\n",
      "Iter 78080, Minibatch Loss= 306.896912, Valid Accuracy= 0.79688\n",
      "Iter 79360, Minibatch Loss= 35.815590, Training Accuracy= 0.97656\n",
      "Iter 79360, Minibatch Loss= 360.005981, Valid Accuracy= 0.75000\n",
      "Iter 80640, Minibatch Loss= 43.800785, Training Accuracy= 0.97656\n",
      "Iter 80640, Minibatch Loss= 367.247192, Valid Accuracy= 0.78125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 81920, Minibatch Loss= 24.236622, Training Accuracy= 0.98438\n",
      "Iter 81920, Minibatch Loss= 355.462097, Valid Accuracy= 0.76562\n",
      "Iter 83200, Minibatch Loss= 33.782822, Training Accuracy= 0.98438\n",
      "Iter 83200, Minibatch Loss= 309.026581, Valid Accuracy= 0.79688\n",
      "Iter 84480, Minibatch Loss= 35.111893, Training Accuracy= 0.98438\n",
      "Iter 84480, Minibatch Loss= 292.739319, Valid Accuracy= 0.79688\n",
      "Iter 85760, Minibatch Loss= 31.301935, Training Accuracy= 0.96875\n",
      "Iter 85760, Minibatch Loss= 295.591064, Valid Accuracy= 0.76562\n",
      "Iter 87040, Minibatch Loss= 15.666517, Training Accuracy= 0.97656\n",
      "Iter 87040, Minibatch Loss= 318.450500, Valid Accuracy= 0.76562\n",
      "Iter 88320, Minibatch Loss= 33.446156, Training Accuracy= 0.97656\n",
      "Iter 88320, Minibatch Loss= 211.227997, Valid Accuracy= 0.82812\n",
      "Iter 89600, Minibatch Loss= 30.718134, Training Accuracy= 0.97656\n",
      "Iter 89600, Minibatch Loss= 220.929459, Valid Accuracy= 0.79688\n",
      "Iter 90880, Minibatch Loss= 22.606049, Training Accuracy= 0.97656\n",
      "Iter 90880, Minibatch Loss= 347.274628, Valid Accuracy= 0.76562\n",
      "Iter 92160, Minibatch Loss= 21.397343, Training Accuracy= 0.97656\n",
      "Iter 92160, Minibatch Loss= 377.733002, Valid Accuracy= 0.75000\n",
      "Iter 93440, Minibatch Loss= 12.117676, Training Accuracy= 0.98438\n",
      "Iter 93440, Minibatch Loss= 213.573486, Valid Accuracy= 0.82812\n",
      "Iter 94720, Minibatch Loss= 22.882448, Training Accuracy= 0.98438\n",
      "Iter 94720, Minibatch Loss= 186.203186, Valid Accuracy= 0.90625\n",
      "Iter 96000, Minibatch Loss= 9.071417, Training Accuracy= 0.99219\n",
      "Iter 96000, Minibatch Loss= 266.339142, Valid Accuracy= 0.85938\n",
      "Iter 97280, Minibatch Loss= 18.506880, Training Accuracy= 0.97656\n",
      "Iter 97280, Minibatch Loss= 228.891220, Valid Accuracy= 0.82812\n",
      "Iter 98560, Minibatch Loss= 6.685570, Training Accuracy= 0.99219\n",
      "Iter 98560, Minibatch Loss= 243.735184, Valid Accuracy= 0.81250\n",
      "Iter 99840, Minibatch Loss= 21.352562, Training Accuracy= 0.98438\n",
      "Iter 99840, Minibatch Loss= 184.860565, Valid Accuracy= 0.85938\n",
      "Iter 101120, Minibatch Loss= 1.117802, Training Accuracy= 0.98438\n",
      "Iter 101120, Minibatch Loss= 198.339325, Valid Accuracy= 0.82812\n",
      "Iter 102400, Minibatch Loss= 27.128380, Training Accuracy= 0.98438\n",
      "Iter 102400, Minibatch Loss= 247.147293, Valid Accuracy= 0.81250\n",
      "Iter 103680, Minibatch Loss= 22.592472, Training Accuracy= 0.98438\n",
      "Iter 103680, Minibatch Loss= 313.928619, Valid Accuracy= 0.82812\n",
      "Iter 104960, Minibatch Loss= 9.665324, Training Accuracy= 0.98438\n",
      "Iter 104960, Minibatch Loss= 214.322678, Valid Accuracy= 0.85938\n",
      "Iter 106240, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 106240, Minibatch Loss= 125.776695, Valid Accuracy= 0.89062\n",
      "Iter 107520, Minibatch Loss= 21.902658, Training Accuracy= 0.99219\n",
      "Iter 107520, Minibatch Loss= 172.025833, Valid Accuracy= 0.85938\n",
      "Iter 108800, Minibatch Loss= 21.516159, Training Accuracy= 0.99219\n",
      "Iter 108800, Minibatch Loss= 233.450836, Valid Accuracy= 0.81250\n",
      "Iter 110080, Minibatch Loss= 2.692600, Training Accuracy= 0.99219\n",
      "Iter 110080, Minibatch Loss= 257.282104, Valid Accuracy= 0.82812\n",
      "Iter 111360, Minibatch Loss= 11.691980, Training Accuracy= 0.98438\n",
      "Iter 111360, Minibatch Loss= 150.469406, Valid Accuracy= 0.87500\n",
      "Iter 112640, Minibatch Loss= 29.355055, Training Accuracy= 0.98438\n",
      "Iter 112640, Minibatch Loss= 95.553604, Valid Accuracy= 0.92188\n",
      "Iter 113920, Minibatch Loss= 1.246277, Training Accuracy= 0.99219\n",
      "Iter 113920, Minibatch Loss= 182.218536, Valid Accuracy= 0.85938\n",
      "Iter 115200, Minibatch Loss= 11.819128, Training Accuracy= 0.98438\n",
      "Iter 115200, Minibatch Loss= 161.650482, Valid Accuracy= 0.84375\n",
      "Iter 116480, Minibatch Loss= 10.855625, Training Accuracy= 0.99219\n",
      "Iter 116480, Minibatch Loss= 145.861755, Valid Accuracy= 0.85938\n",
      "Iter 117760, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 117760, Minibatch Loss= 78.805420, Valid Accuracy= 0.90625\n",
      "Iter 119040, Minibatch Loss= 27.759853, Training Accuracy= 0.99219\n",
      "Iter 119040, Minibatch Loss= 109.872131, Valid Accuracy= 0.87500\n",
      "Iter 120320, Minibatch Loss= 9.516880, Training Accuracy= 0.99219\n",
      "Iter 120320, Minibatch Loss= 154.245911, Valid Accuracy= 0.84375\n",
      "Iter 121600, Minibatch Loss= 6.479316, Training Accuracy= 0.99219\n",
      "Iter 121600, Minibatch Loss= 135.416077, Valid Accuracy= 0.85938\n",
      "Iter 122880, Minibatch Loss= 13.055698, Training Accuracy= 0.98438\n",
      "Iter 122880, Minibatch Loss= 93.275963, Valid Accuracy= 0.89062\n",
      "Iter 124160, Minibatch Loss= 12.460915, Training Accuracy= 0.99219\n",
      "Iter 124160, Minibatch Loss= 111.083664, Valid Accuracy= 0.87500\n",
      "Iter 125440, Minibatch Loss= 36.044483, Training Accuracy= 0.98438\n",
      "Iter 125440, Minibatch Loss= 175.852783, Valid Accuracy= 0.82812\n",
      "Iter 126720, Minibatch Loss= 9.374886, Training Accuracy= 0.99219\n",
      "Iter 126720, Minibatch Loss= 210.786499, Valid Accuracy= 0.82812\n",
      "Iter 128000, Minibatch Loss= 13.660698, Training Accuracy= 0.98438\n",
      "Iter 128000, Minibatch Loss= 114.946472, Valid Accuracy= 0.87500\n",
      "Iter 129280, Minibatch Loss= 13.736794, Training Accuracy= 0.98438\n",
      "Iter 129280, Minibatch Loss= 46.916183, Valid Accuracy= 0.95312\n",
      "Iter 130560, Minibatch Loss= 5.629822, Training Accuracy= 0.99219\n",
      "Iter 130560, Minibatch Loss= 114.671402, Valid Accuracy= 0.89062\n",
      "Iter 131840, Minibatch Loss= 13.645452, Training Accuracy= 0.98438\n",
      "Iter 131840, Minibatch Loss= 145.943359, Valid Accuracy= 0.89062\n",
      "Iter 133120, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 133120, Minibatch Loss= 126.674469, Valid Accuracy= 0.87500\n",
      "Iter 134400, Minibatch Loss= 6.374865, Training Accuracy= 0.99219\n",
      "Iter 134400, Minibatch Loss= 73.720383, Valid Accuracy= 0.93750\n",
      "Iter 135680, Minibatch Loss= 3.832544, Training Accuracy= 0.99219\n",
      "Iter 135680, Minibatch Loss= 91.997231, Valid Accuracy= 0.89062\n",
      "Iter 136960, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 136960, Minibatch Loss= 94.304031, Valid Accuracy= 0.87500\n",
      "Iter 138240, Minibatch Loss= 17.110786, Training Accuracy= 0.99219\n",
      "Iter 138240, Minibatch Loss= 160.308990, Valid Accuracy= 0.85938\n",
      "Iter 139520, Minibatch Loss= 2.286556, Training Accuracy= 0.98438\n",
      "Iter 139520, Minibatch Loss= 97.282059, Valid Accuracy= 0.87500\n",
      "Iter 140800, Minibatch Loss= 15.205896, Training Accuracy= 0.98438\n",
      "Iter 140800, Minibatch Loss= 34.884079, Valid Accuracy= 0.92188\n",
      "Iter 142080, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 142080, Minibatch Loss= 83.273613, Valid Accuracy= 0.89062\n",
      "Iter 143360, Minibatch Loss= 2.768257, Training Accuracy= 0.98438\n",
      "Iter 143360, Minibatch Loss= 163.484100, Valid Accuracy= 0.82812\n",
      "Iter 144640, Minibatch Loss= 16.104998, Training Accuracy= 0.98438\n",
      "Iter 144640, Minibatch Loss= 112.488197, Valid Accuracy= 0.84375\n",
      "Iter 145920, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 145920, Minibatch Loss= 36.772659, Valid Accuracy= 0.95312\n",
      "Iter 147200, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 147200, Minibatch Loss= 50.144127, Valid Accuracy= 0.93750\n",
      "Iter 148480, Minibatch Loss= 20.392797, Training Accuracy= 0.98438\n",
      "Iter 148480, Minibatch Loss= 99.971550, Valid Accuracy= 0.90625\n",
      "Iter 149760, Minibatch Loss= 9.133162, Training Accuracy= 0.98438\n",
      "Iter 149760, Minibatch Loss= 160.892471, Valid Accuracy= 0.85938\n",
      "Iter 151040, Minibatch Loss= 1.427269, Training Accuracy= 0.99219\n",
      "Iter 151040, Minibatch Loss= 76.518074, Valid Accuracy= 0.93750\n",
      "Iter 152320, Minibatch Loss= 1.011589, Training Accuracy= 0.99219\n",
      "Iter 152320, Minibatch Loss= 29.088570, Valid Accuracy= 0.95312\n",
      "Iter 153600, Minibatch Loss= 1.825680, Training Accuracy= 0.99219\n",
      "Iter 153600, Minibatch Loss= 57.844681, Valid Accuracy= 0.93750\n",
      "Iter 154880, Minibatch Loss= 0.753162, Training Accuracy= 0.98438\n",
      "Iter 154880, Minibatch Loss= 106.511909, Valid Accuracy= 0.89062\n",
      "Iter 156160, Minibatch Loss= 1.459604, Training Accuracy= 0.99219\n",
      "Iter 156160, Minibatch Loss= 99.199280, Valid Accuracy= 0.90625\n",
      "Iter 157440, Minibatch Loss= 2.006889, Training Accuracy= 0.99219\n",
      "Iter 157440, Minibatch Loss= 31.650024, Valid Accuracy= 0.93750\n",
      "Iter 158720, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 158720, Minibatch Loss= 53.173943, Valid Accuracy= 0.95312\n",
      "Iter 160000, Minibatch Loss= 1.828981, Training Accuracy= 0.99219\n",
      "Iter 160000, Minibatch Loss= 110.511551, Valid Accuracy= 0.90625\n",
      "Iter 161280, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 161280, Minibatch Loss= 119.625412, Valid Accuracy= 0.90625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 162560, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 162560, Minibatch Loss= 62.986073, Valid Accuracy= 0.95312\n",
      "Iter 163840, Minibatch Loss= 0.897896, Training Accuracy= 0.99219\n",
      "Iter 163840, Minibatch Loss= 30.798271, Valid Accuracy= 0.95312\n",
      "Iter 165120, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 165120, Minibatch Loss= 66.809166, Valid Accuracy= 0.92188\n",
      "Iter 166400, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 166400, Minibatch Loss= 118.771393, Valid Accuracy= 0.90625\n",
      "Iter 167680, Minibatch Loss= 2.976720, Training Accuracy= 0.99219\n",
      "Iter 167680, Minibatch Loss= 92.248032, Valid Accuracy= 0.87500\n",
      "Iter 168960, Minibatch Loss= 7.689154, Training Accuracy= 0.98438\n",
      "Iter 168960, Minibatch Loss= 31.878410, Valid Accuracy= 0.93750\n",
      "Iter 170240, Minibatch Loss= 13.438980, Training Accuracy= 0.99219\n",
      "Iter 170240, Minibatch Loss= 46.828468, Valid Accuracy= 0.93750\n",
      "Iter 171520, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 171520, Minibatch Loss= 99.685036, Valid Accuracy= 0.92188\n",
      "Iter 172800, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 172800, Minibatch Loss= 124.350868, Valid Accuracy= 0.92188\n",
      "Iter 174080, Minibatch Loss= 2.208576, Training Accuracy= 0.98438\n",
      "Iter 174080, Minibatch Loss= 49.261021, Valid Accuracy= 0.89062\n",
      "Iter 175360, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 175360, Minibatch Loss= 20.297394, Valid Accuracy= 0.95312\n",
      "Iter 176640, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 176640, Minibatch Loss= 45.323837, Valid Accuracy= 0.93750\n",
      "Iter 177920, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 177920, Minibatch Loss= 34.175968, Valid Accuracy= 0.93750\n",
      "Iter 179200, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 179200, Minibatch Loss= 32.548031, Valid Accuracy= 0.95312\n",
      "Iter 180480, Minibatch Loss= 2.592171, Training Accuracy= 0.98438\n",
      "Iter 180480, Minibatch Loss= 13.711173, Valid Accuracy= 0.96875\n",
      "Iter 181760, Minibatch Loss= 5.687826, Training Accuracy= 0.99219\n",
      "Iter 181760, Minibatch Loss= 42.272633, Valid Accuracy= 0.96875\n",
      "Iter 183040, Minibatch Loss= 1.543888, Training Accuracy= 0.99219\n",
      "Iter 183040, Minibatch Loss= 47.124466, Valid Accuracy= 0.95312\n",
      "Iter 184320, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 184320, Minibatch Loss= 46.463615, Valid Accuracy= 0.93750\n",
      "Iter 185600, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 185600, Minibatch Loss= 29.186762, Valid Accuracy= 0.95312\n",
      "Iter 186880, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 186880, Minibatch Loss= 11.298605, Valid Accuracy= 0.98438\n",
      "Iter 188160, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 188160, Minibatch Loss= 27.914295, Valid Accuracy= 0.93750\n",
      "Iter 189440, Minibatch Loss= 3.010926, Training Accuracy= 0.99219\n",
      "Iter 189440, Minibatch Loss= 62.003712, Valid Accuracy= 0.92188\n",
      "Iter 190720, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 190720, Minibatch Loss= 51.677696, Valid Accuracy= 0.93750\n",
      "Iter 192000, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 192000, Minibatch Loss= 21.998791, Valid Accuracy= 0.95312\n",
      "Iter 193280, Minibatch Loss= 1.531238, Training Accuracy= 0.99219\n",
      "Iter 193280, Minibatch Loss= 13.669775, Valid Accuracy= 0.96875\n",
      "Iter 194560, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 194560, Minibatch Loss= 14.050796, Valid Accuracy= 0.95312\n",
      "Iter 195840, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 195840, Minibatch Loss= 23.497993, Valid Accuracy= 0.95312\n",
      "Iter 197120, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 197120, Minibatch Loss= 28.859980, Valid Accuracy= 0.95312\n",
      "Iter 198400, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 198400, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Iter 199680, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 199680, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Iter 200960, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 200960, Minibatch Loss= 11.745041, Valid Accuracy= 0.95312\n",
      "Iter 202240, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 202240, Minibatch Loss= 19.113098, Valid Accuracy= 0.95312\n",
      "Iter 203520, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 203520, Minibatch Loss= 0.402900, Valid Accuracy= 0.96875\n",
      "Iter 204800, Minibatch Loss= 0.338104, Training Accuracy= 0.99219\n",
      "Iter 204800, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Iter 206080, Minibatch Loss= 0.855643, Training Accuracy= 0.99219\n",
      "Iter 206080, Minibatch Loss= 8.250280, Valid Accuracy= 0.98438\n",
      "Iter 207360, Minibatch Loss= 4.492865, Training Accuracy= 0.98438\n",
      "Iter 207360, Minibatch Loss= 10.102399, Valid Accuracy= 0.95312\n",
      "Iter 208640, Minibatch Loss= 2.236984, Training Accuracy= 0.99219\n",
      "Iter 208640, Minibatch Loss= 34.375278, Valid Accuracy= 0.95312\n",
      "Iter 209920, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 209920, Minibatch Loss= 0.149935, Valid Accuracy= 0.98438\n",
      "Iter 211200, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 211200, Minibatch Loss= 11.908973, Valid Accuracy= 0.96875\n",
      "Iter 212480, Minibatch Loss= 1.759537, Training Accuracy= 0.99219\n",
      "Iter 212480, Minibatch Loss= 29.621471, Valid Accuracy= 0.95312\n",
      "Iter 213760, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 213760, Minibatch Loss= 14.141263, Valid Accuracy= 0.96875\n",
      "Iter 215040, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 215040, Minibatch Loss= 6.253181, Valid Accuracy= 0.98438\n",
      "Iter 216320, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 216320, Minibatch Loss= 5.759586, Valid Accuracy= 0.98438\n",
      "Iter 217600, Minibatch Loss= 1.972392, Training Accuracy= 0.99219\n",
      "Iter 217600, Minibatch Loss= 29.167496, Valid Accuracy= 0.96875\n",
      "Iter 218880, Minibatch Loss= 0.002024, Training Accuracy= 1.00000\n",
      "Iter 218880, Minibatch Loss= 50.920902, Valid Accuracy= 0.95312\n",
      "Iter 220160, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 220160, Minibatch Loss= 4.282719, Valid Accuracy= 0.98438\n",
      "Iter 221440, Minibatch Loss= 0.734896, Training Accuracy= 0.99219\n",
      "Iter 221440, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Iter 222720, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 222720, Minibatch Loss= 32.604767, Valid Accuracy= 0.95312\n",
      "Iter 224000, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 224000, Minibatch Loss= 34.064690, Valid Accuracy= 0.95312\n",
      "Iter 225280, Minibatch Loss= 2.181018, Training Accuracy= 0.99219\n",
      "Iter 225280, Minibatch Loss= 16.323370, Valid Accuracy= 0.96875\n",
      "Iter 226560, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 226560, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Iter 227840, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 227840, Minibatch Loss= 7.821575, Valid Accuracy= 0.98438\n",
      "Iter 229120, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 229120, Minibatch Loss= 24.329336, Valid Accuracy= 0.98438\n",
      "Iter 230400, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 230400, Minibatch Loss= 26.761938, Valid Accuracy= 0.96875\n",
      "Iter 231680, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 231680, Minibatch Loss= 16.260168, Valid Accuracy= 0.98438\n",
      "Iter 232960, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 232960, Minibatch Loss= 14.071087, Valid Accuracy= 0.98438\n",
      "Iter 234240, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 234240, Minibatch Loss= 45.762390, Valid Accuracy= 0.95312\n",
      "Iter 235520, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 235520, Minibatch Loss= 44.391762, Valid Accuracy= 0.93750\n",
      "Iter 236800, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 236800, Minibatch Loss= 35.120178, Valid Accuracy= 0.93750\n",
      "Iter 238080, Minibatch Loss= 1.621748, Training Accuracy= 0.98438\n",
      "Iter 238080, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Iter 239360, Minibatch Loss= 7.407971, Training Accuracy= 0.99219\n",
      "Iter 239360, Minibatch Loss= 24.209398, Valid Accuracy= 0.96875\n",
      "Iter 240640, Minibatch Loss= 4.014008, Training Accuracy= 0.99219\n",
      "Iter 240640, Minibatch Loss= 29.330231, Valid Accuracy= 0.95312\n",
      "Iter 241920, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 241920, Minibatch Loss= 23.649555, Valid Accuracy= 0.98438\n",
      "Iter 243200, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 243200, Minibatch Loss= 11.702423, Valid Accuracy= 0.98438\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 244480, Minibatch Loss= 0.346364, Training Accuracy= 0.99219\n",
      "Iter 244480, Minibatch Loss= 4.908966, Valid Accuracy= 0.98438\n",
      "Iter 245760, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 245760, Minibatch Loss= 22.986813, Valid Accuracy= 0.98438\n",
      "Iter 247040, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 247040, Minibatch Loss= 12.641784, Valid Accuracy= 0.95312\n",
      "Iter 248320, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 248320, Minibatch Loss= 9.605269, Valid Accuracy= 0.96875\n",
      "Iter 249600, Minibatch Loss= 0.000000, Training Accuracy= 1.00000\n",
      "Iter 249600, Minibatch Loss= 0.000000, Valid Accuracy= 1.00000\n",
      "Optimization Finished!\n",
      "Testing Accuracy: 0.84765625\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    sess.run(init)\n",
    "\n",
    "    step = 1\n",
    "\n",
    "    # Keep training until reach max iterations\n",
    "\n",
    "    print(\"train_try\")\n",
    "\n",
    "    while step * batch_size < training_iters:\n",
    "\n",
    "        # print(\"train_try\")\n",
    "\n",
    "        #my_way\n",
    "\n",
    "        for minibatch_index in range(n_train_batches):\n",
    "\n",
    "            \n",
    "\n",
    "            index = minibatch_index\n",
    "\n",
    "            index_l = index * batch_size\n",
    "\n",
    "            index_h = (index + 1) * batch_size\n",
    "\n",
    "\n",
    "\n",
    "            sess.run(optimizer, feed_dict={x: train_set_x.eval()[index_l: index_h], y: train_set_y.eval()[index_l: index_h],\n",
    "\n",
    "                                           keep_prob: dropout})\n",
    "\n",
    "            if valid_num == 10:\n",
    "\n",
    "                sess.run(optimizer, feed_dict={x: valid_set_x.eval()[int(index_l/2): int(index_h/2)], y: valid_set_y.eval()[int(index_l/2): int(index_h/2)],\n",
    "\n",
    "                                           keep_prob: dropout})\n",
    "\n",
    "                valid_num = 1\n",
    "\n",
    "\n",
    "\n",
    "            if step % display_step == 0:\n",
    "\n",
    "                # Calculate batch loss and accuracy\n",
    "\n",
    "                loss, acc = sess.run([cost, accuracy], feed_dict={x: train_set_x.eval()[index_l: index_h],\n",
    "\n",
    "                                                                  y: train_set_y.eval()[index_l: index_h],\n",
    "\n",
    "                                                                  keep_prob: 1.})\n",
    "\n",
    "                print(\"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "\n",
    "                      \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "\n",
    "                      \"{:.5f}\".format(acc))\n",
    "\n",
    "                loss, acc = sess.run([cost, accuracy], feed_dict={x: valid_set_x.eval()[int(index_l/2): int(index_h/2)],\n",
    "\n",
    "                                                                  y: valid_set_y.eval()[int(index_l/2): int(index_h/2)],\n",
    "\n",
    "                                                                  keep_prob: 1.})\n",
    "\n",
    "                print(\"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "\n",
    "                      \"{:.6f}\".format(loss) + \", Valid Accuracy= \" + \\\n",
    "\n",
    "                      \"{:.5f}\".format(acc))\n",
    "\n",
    "            step += 1\n",
    "\n",
    "            valid_num += 1\n",
    "\n",
    "\n",
    "\n",
    "    print(\"Optimization Finished!\")\n",
    "\n",
    "\n",
    "\n",
    "    # Calculate accuracy for 256 mnist test images\n",
    "\n",
    "    print(\"Testing Accuracy:\", \\\n",
    "\n",
    "        sess.run(accuracy, feed_dict={x: test_set_x.eval()[:256],\n",
    "\n",
    "                                      y: test_set_y.eval()[:256],\n",
    "\n",
    "                                      keep_prob: 1.}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###########################################################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When\n",
    "\n",
    "'wc1': tf.Variable(tf.random_normal([3, 3, 1, 32]))\n",
    "\n",
    "### Accuracy = 0.87109375"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When \n",
    "\n",
    "learning rate = 0.1 ( increased from 0.001 to 0.1 )\n",
    "\n",
    "### Accuracy =  0.125"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When\n",
    "\n",
    "learning_rate = 0.1\n",
    "\n",
    "training_iters = 2500\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "display_step = 15\n",
    "\n",
    "### Accuracy = 0.078125"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When\n",
    "\n",
    "training_iters = 2500\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32]))\n",
    "\n",
    "'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64]))\n",
    "\n",
    "### Accuracy = 0.14453125"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Max Accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## When \n",
    "learning_rate = 0.001\n",
    "\n",
    "training_iters = 250000\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "display_step = 10\n",
    "\n",
    "### Accurcy = 0.86328"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We get the Highest Accuracy when we INCREASE the ITERATION and Decrease the LEARNING RATE\n",
    "### From this we get to know that the Deep Learning needs more Data as well as higer steps to learn the maximum features of the DATA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
